import LiveJournalParser;
import OutputDataParser;
import CrawlerConfig;

import sys;
import copy;

class LiveJournalBlogGraphCrawler:
    def __init__(self):
        self.urlFilter = LiveJournalParser.LiveJournalUrlFilter();
        self.urlConverter = LiveJournalParser.LiveJournalUrlConverter();
        self.communityProfileParser = LiveJournalParser.LiveJournalCommunityProfileParser();
        self.userProfileParser = LiveJournalParser.LiveJournalUserProfileParser();
        
    
    def init(self, nameFilePath, communityFilePath, userFilePath, communityshipFilePath, friendshipFilePath):    
        self.crawledCommunities, self.crawledUsers = self.recover(communityFilePath, userFilePath);
        self.communityFile = open(communityFilePath, 'a');
        self.userFile = open(userFilePath, 'a');
        self.communityshipFile = open(communityshipFilePath, 'a');
        self.friendshipFile = open(friendshipFilePath, 'a');

        print 'LiveJournalBlogGraphCrawler::initial() starting......';
        communities = set();
        users = set();
        cnt = 0;
        prevLen = 0;
        nameFile = open(nameFilePath);
        names = map(str.strip, nameFile.readlines());
        for name in names:
            tokens = name.split(':');
            if tokens[0] == 'user':
                users.add(tokens[1]);
            elif tokens[0] == 'community':
                communities.add(tokens[1]);
        print 'LiveJournalBlogGraphCrawler::initial() over: ', len(communities), 'communities and', len(users), 'users';    
        return communities.difference(self.crawledCommunities), users.difference(self.crawledUsers);

    def recover(self, communityFilePath, userFilePath):
        print 'recovering from', communityFilePath, 'and', userFilePath;
        communities = self.loadSet(communityFilePath);
        users = self.loadSet(userFilePath);
        return communities, users;

    def loadSet(self, filePath):
        '''
        load set from a file, and each line is an element 
        '''
        file = open(filePath);
        lines = file.readlines();
        file.close();
        return set(map(str.strip, lines));
        
    
    
    def clear(self):
        self.communityFile.close();
        self.userFile.close();
        self.communityshipFile.close();
        self.friendshipFile.close();
                
    '''
        get all users in the community set
    '''
    def processCommunity(self, communities):
        print 'LiveJournalBlogGraphCrawler::processCommunity() starting';
        newusers = set();
        for name in communities:
            self.communityFile.write(name + '\n');
            communityships = self.extractCommunityship(name);
            print len(communityships), 'users in community', name;
            for communityship in communityships:
                if self.crawledUsers.__contains__(communityship):
                    continue;
                newusers.add(communityship);
                self.communityshipFile.write(name + ':' + str(communityship) + '\n');
        print 'LiveJournalBlogGraphCrawler::processCommunity() ending:', len(newusers), 'new users!';
        return newusers;
                
    '''
        get all users in a community
    '''
    def extractCommunityship(self, name):
        homeUrl = self.urlConverter.commuityName2Home(name);
        profileUrl = self.urlConverter.home2Profile(homeUrl);
        communityProfile = self.communityProfileParser.parseUrl(profileUrl);
        if communityProfile:
            return communityProfile.communityships;
        else:
            return [];
            
    '''
        extract the friendship from a user set seed
    '''
    def processUser(self, users):
        print 'LiveJournalBlogGraphCrawler::processUser() starting:', len(users), 'users!';
        currUsers = copy.deepcopy(users);
        newUsers = set();
        while currUsers.__len__():
            for name in currUsers:
                friendships = self.extractFriendship(name);
                self.userFile.write(name + '\n');
                for friendship in friendships:
                    self.friendshipFile.write(name + ':' + str(friendship) + '\n');
                    if not self.crawledUsers.__contains__(friendship):
                        self.crawledUsers.add(friendship);
                        newUsers.add(friendship);
                print name, 'has', len(friendships), 'friends';
                print 'There are', len(newUsers), 'new users!';
            currUsers = copy.deepcopy(newUsers);
        print 'LiveJournalBlogGraphCrawler::processUser() ending!';
       
    '''
        get all friends of a user
    '''            
    def extractFriendship(self, name):
        homeUrl = self.urlConverter.userName2Home(name);
        profileUrl = self.urlConverter.home2Profile(homeUrl);
        userProfile = self.userProfileParser.parseUrl(profileUrl);
        if userProfile:
            return userProfile.friendships;
        else:
            return [];
        
    def loadNamesFromPostings(self, postingFilePaths, nameFilePath):
        nameFile = open(nameFilePath, 'w');
        postingFile = OutputDataParser.PostingFile();
        cnt = 0;
        for postingFilePath in postingFilePaths:
            postingFile.open(postingFilePath);
            while postingFile.next() >= 0:
                posting = postingFile.getCurrElement();
                postingUrl = posting.url;
                if not self.urlFilter.isValid(postingUrl):
                    continue;
                homeUrl = self.urlConverter.posting2Home(postingUrl);
                name = self.urlConverter.getName(homeUrl);
                if self.urlConverter.isCommunity(homeUrl):
                    nameFile.write("community:" + name + '\n');
                if self.urlConverter.isUser(homeUrl):
                    nameFile.write("user:" + name + '\n');
                cnt += 1;
                if cnt % 2000 == 0:
                    print str(cnt);
                    sys.stdout.flush();    

    def collectNames(self, redundancyNameFilePath, cleanNameFilePath):
        redundancyNameFile = open(redundancyNameFilePath);
        cleanNameFile = open(cleanNameFilePath, 'w');
        cleanNames = set();
        line = redundancyNameFile.readline();
        while line:
            cleanNames.add(line.strip()); 
            line = redundancyNameFile.readline();
        redundancyNameFile.close();
        for name in cleanNames:
            cleanNameFile.write(name + '\n');
        cleanNameFile.close();
    
    def run(self, nameFilePath, communityFilePath, userFilePath, communityshipFilePath, friendshipFilePath):
        communities, users = self.init(nameFilePath, communityFilePath, userFilePath, communityshipFilePath, friendshipFilePath);
        newusers = self.processCommunity(communities);
        users = users.union(newusers);
        self.processUser(users);
        self.clear();
    
                   
if __name__ == '__main__':
    argc = len(sys.argv);
    if argc >= 2:
        option = sys.argv[1];
        if option == '--loadnames':
            postingFilePaths = [sys.argv[2]];
            nameFilePath = sys.argv[3];
            crawler = LiveJournalBlogGraphCrawler();
            crawler.loadNamesFromPostings(postingFilePaths, nameFilePath);
        elif option == '--collectnames':
            redundancyNameFilePath = sys.argv[2];
            cleanNameFilePath = sys.argv[3];
            crawler = LiveJournalBlogGraphCrawler();
            crawler.collectNames(redundancyNameFilePath, cleanNameFilePath);
        elif option == '--crawl':
            configFilePath = sys.argv[2];
            config = CrawlerConfig.BlogGraphCrawlerConfig(configFilePath);
            crawler = LiveJournalBlogGraphCrawler();
            crawler.run(config.data[config.NAME_FILE_PATH_VARNAME],
                                          config.data[config.COMMUNITY_FILE_PATH_VARNAME],
                                          config.data[config.USER_FILE_PATH_VARNAME],
                                          config.data[config.COMMUNITYSHIP_FILE_PATH_VARNAME],
                                          config.data[config.FRIENDSHIP_FILE_PATH_VARNAME]);
